unit uDZPagedDataSet;

interface

uses
  SysUtils, Classes, uDZSysUtils, uDZPersistent, uDZThread,
  uDZLinkList, uDZBinaryTree, uDZHashTable, uDZDynamicArray;

type
  TDzPagedDataSet = class;

  TDataType = (dtUnknown, dtAnsiStr, dtWideStr, dtBoolean, dtSmallInt, dtWord,
    dtInteger, dtInt64, dtFloat, dtCurrency, dtDate, dtTime, dtDateTime);
    
  TDzDataPageEntry = record
    Index: Integer;
    Data: TDzPersistentList;
  end;
  PDzDataPageEntry = ^TDzDataPageEntry;

  TDzHashIndex = record
    Name: TNameString;
    DataType: TDataType;
    Table: TDzContainer;
  end;
  PDzHashIndex = ^TDzHashIndex;

  PIntegerLink = ^TIntegerLink;
  TIntegerLink = record
    FLink: PIntegerLink;
    Value: Integer;
  end;
  
  TDzPagedDataset = class
  private
    fPageIndex: Integer;
    fPageSize: Integer;
    fPages: TDzIntegerRBTree;
    fPageCount: Integer;
    fRecordCount: Integer;
    fLoadingPages: PIntegerLink;
    fNeededPages: PIntegerLink;  
    fIndexes: TList;
    fInsertedIndexes: TList;
    fInserted: TDzPersistentList;
    CriticalSection: TDzCriticalSection;
    function GetPage(Index: Integer): TDzPersistentList;
    procedure SetPage(Index: Integer; const Value: TDzPersistentList);
    function GetItem(Index: Integer): TDzPersistent;
    procedure SetItem(Index: Integer; const Value: TDzPersistent);
    function GetItemPageIndex(ItemIndex: Integer): Integer;

    procedure AddIndex(const Name: TNameString; DataType: TDataType;
      var IndexList: TList);

    function HashFindItem(IndexList: TList; const KeyField: TNameString;
      const KeyValue: Variant): TDzPersistent;

    procedure DeleteInsertedItem(Index: PDzHashIndex; const KeyField: TNameString;
      const KeyValue: Variant);
      
    procedure EmptyIndex(var IndexList: TList);
  protected
    procedure DeleteRecords(list: TDzPersistentList);
    procedure AddRecords(list: TDzPersistentList);
    procedure EmptyIndexes;
    procedure EmptyInsertedIndexes;
    procedure FreeIndexes;
    function FindIndex(IndexList: TList; const Name: TNameString): PDzHashIndex;
    procedure FetchPage(PageIndex: Integer); virtual; abstract;
    procedure RecordAdded(data: TDzPersistentList); dynamic;
    procedure RecordDeleted(data: TDzPersistentList); dynamic;
    procedure DeleteFromInserted(Item: TDzPersistent); overload;
  public
    constructor Create;
    destructor Destroy; override;
    procedure Lock;
    procedure Release;
    procedure AddIndexField(const Name: TNameString; DataType: TDataType);
    procedure EnableIndex;
    function IndexActive: Boolean;

    function FindItem(const KeyField: TNameString;
      const KeyValue: Variant): TDzPersistent;

    function FindInsertedItem(const KeyField: TNameString;
      const KeyValue: Variant): TDzPersistent;

    function PageLoading(PageIndex: Integer): Boolean;
    procedure PageLoaded(PageIndex: Integer);
    
    procedure Clear;
    procedure RefreshPage(PageIndex: Integer);
    procedure ForcePages(StartIndex, Count: Integer; AbortOtherPages: Boolean);
    procedure LoadNeeded;

    procedure AddInserted(Item: TDzPersistent);
    procedure ItemDeleted(Deled: TDzPersistent); overload;
    procedure ItemDeleted(const KeyField: TNameString; const KeyValue: Variant); overload;

    property PageSize: Integer read fPageSize write fPageSize;
    property PageCount: Integer read fPageCount write fPageCount;
    property Pages[Index: Integer]: TDzPersistentList read GetPage write SetPage;
    property RecordCount: Integer read fRecordCount write fRecordCount;
    property Item[Index: Integer]: TDzPersistent read GetItem write SetItem;
    property ItemPageIndex[ItemIndex: Integer]: Integer read GetItemPageIndex;
    property Inserted: TDzPersistentList read fInserted;
  end;

implementation

function ComparePageEntry(Item1, Item2: Pointer): Integer;
begin
  Result := PDzDataPageEntry(Item1).Index - PDzDataPageEntry(Item2).Index;
end;

procedure IntegerLinkList_Clear(var pList: PIntegerLink);
var
  pNode: PIntegerLink;
begin
  while Assigned(pList) do
  begin
    pNode := pList.FLink;
    Dispose(pList);
    pList := pNode;
  end;
end;

procedure IntegerLinkList_Insert(var pList: PIntegerLink; Value: Integer);
var
  pNode: PIntegerLink;
begin
  New(pNode);
  pNode.Value := Value;
  pNode.FLink := pList;
  pList := pNode; 
end;

function IntegerLinkList_Delete(var pList: PIntegerLink; Value: Integer): Boolean;
var
  pNode, FNode: PIntegerLink;
begin
  Result := False;
  FNode := nil;
  pNode := pList;
  while Assigned(pNode) do
  begin
    if pNode.Value = Value then
    begin
      if Assigned(FNode) then FNode.FLink := pNode.FLink
      else pList := pNode.FLink;
      Dispose(pNode);
      Result := True;
      Exit;
    end
    else begin
      FNode := pNode;
      pNode := pNode.FLink;
    end;
  end;
end;

function IntegerLinkList_Find(pList: PIntegerLink; Value: Integer): Boolean;
var
  pNode: PIntegerLink;
begin
  Result := False;
  pNode := pList;
  while Assigned(pNode) do
  begin
    if pNode.Value = Value then
    begin
      Result := True;
      Exit;
    end
    else pNode := pNode.FLink;
  end;
end;

{ TDzPagedDataset }

procedure TDzPagedDataset.AddInserted(Item: TDzPersistent);
var
  i: Integer;
  Index: PDzHashIndex;
begin
  Lock;
  try
    fInserted.Insert(fInserted.Count, Item);
    if Assigned(fInsertedIndexes) then
    with Item do
    begin
      for i := 0 to fInsertedIndexes.Count - 1 do
      begin
        Index := PDzHashIndex(fInsertedIndexes[i]);
        case Index.DataType of
          dtInteger: (Index.Table as TDzIntegerRBTree).Add(
            Attributes[Index.Name], Item, True);
          dtAnsiStr: (Index.Table as TDzAnsiStrHashTable).Insert(
            Attributes[Index.Name], Item, nil, True);
          dtWideStr: (Index.Table as TDzWideStrHashTable).Insert(
            Attributes[Index.Name], Item, nil, True);
        end;
      end;
    end;
  finally
    Release;
  end;
end;

procedure TDzPagedDataset.AddRecords(list: TDzPersistentList);
var
  i, j: Integer;
  obj: TDzPersistent;
  Index: PDzHashIndex;
begin
  if IndexActive then
  begin
    for i := 0 to list.Count - 1 do
    begin
      obj := list[i];
      for j := 0 to fIndexes.Count - 1 do
      begin
        Index := PDzHashIndex(fIndexes[j]);
        case Index.DataType of
          dtInteger: (Index.Table as TDzIntegerRBTree).Add(
            obj.Attributes[Index.Name], obj, True);
          dtAnsiStr: (Index.Table as TDzAnsiStrHashTable).Insert(
            obj.Attributes[Index.Name], obj, nil, True);
          dtWideStr: (Index.Table as TDzWideStrHashTable).Insert(
            obj.Attributes[Index.Name], obj, nil, True);
        end;
      end;
    end;
    Self.RecordAdded(list);
  end;
end;

procedure TDzPagedDataset.Clear;
var
  i: Integer;
begin
  Lock;
  try
    for i := 0 to fPages.Count - 1 do fPages.List[i].Free;
    fPages.Clear;
    IntegerLinkList_Clear(fLoadingPages);
    IntegerLinkList_Clear(fNeededPages);
    EmptyIndexes;
  finally
    Release;
  end;
end;

procedure TDzPagedDataset.FreeIndexes;
var
  Index: PDzHashIndex;
  i: Integer;
begin
  if fIndexes <> nil then
  begin
    for i := fIndexes.Count - 1 downto 0 do
    begin
      Index := PDzHashIndex(fIndexes[i]);
      Index.Table.Free;
      Dispose(Index);
    end;
    fIndexes.Free;
  end;
  if fInsertedIndexes <> nil then
  begin
    for i := fInsertedIndexes.Count - 1 downto 0 do
    begin
      Index := PDzHashIndex(fInsertedIndexes[i]);
      Index.Table.Free;
      Dispose(Index);
    end;
    fInsertedIndexes.Free;
  end;
end;

procedure TDzPagedDataset.EmptyIndex(var IndexList: TList);
var
  Index: PDzHashIndex;
  i: Integer;
begin
  if IndexList <> nil then
  for i := 0 to IndexList.Count - 1 do
  begin
    Index := PDzHashIndex(IndexList[i]);
    Index.Table.Clear;
  end;
end;

procedure TDzPagedDataset.EmptyIndexes;
begin
  Self.EmptyIndex(fIndexes);
end;

procedure TDzPagedDataset.EmptyInsertedIndexes;
begin
  Self.EmptyIndex(fInsertedIndexes);
end;

constructor TDzPagedDataset.Create;
begin
  CriticalSection := TDzCriticalSection.Create;
  fPages := TDzIntegerRBTree.Create;
  fInserted := TDzPersistentList.Create(TDzPersistent);
  fPageSize := 50;
  fPageIndex := 0;
end;

procedure TDzPagedDataset.AddIndex(const Name: TNameString;
  DataType: TDataType; var IndexList: TList);
var
  Index: PDzHashIndex;
  TableSize: Integer;
begin
  if IndexList = nil then IndexList := TList.Create;
  New(Index);
  Index.Name := Name;
  Index.DataType := DataType;
  TableSize := CalcHashTableSize(1000);
  case Index.DataType of
    dtInteger: Index.Table := TDzIntegerRBTree.Create();
    dtAnsiStr: Index.Table := TDzAnsiStrHashTable.Create(TableSize, nil);
    dtWideStr: Index.Table := TDzWideStrHashTable.Create(TableSize, nil);
  end;
  IndexList.Add(Index);
end;

procedure TDzPagedDataset.AddIndexField(const Name: TNameString;
  DataType: TDataType);
begin
  Self.AddIndex(Name, DataType, fIndexes);
  Self.AddIndex(Name, DataType, fInsertedIndexes);
end;

procedure TDzPagedDataset.DeleteFromInserted(Item: TDzPersistent);
var
  ItemIndex: Integer;
begin
  ItemIndex := fInserted.IndexOf(Item);
  if ItemIndex <> -1 then fInserted.Erase(ItemIndex);
end;

procedure TDzPagedDataset.DeleteInsertedItem(Index: PDzHashIndex;
  const KeyField: TNameString; const KeyValue: Variant);
var
  Item: TDzPersistent;
begin
  case Index.DataType of
    dtInteger:
      with Index.Table as TDzIntegerRBTree do
      begin
        Item := TDzPersistent(Items[KeyValue]);
        if Assigned(Item) then
        begin
          Delete(KeyValue);
          DeleteFromInserted(Item);
        end;
      end;
    dtAnsiStr:
      with Index.Table as TDzAnsiStrHashTable do
      begin
        if Find(KeyValue, @Item) then
        begin
          Delete(KeyValue);
          DeleteFromInserted(Item);
        end;
      end;
    dtWideStr: 
      with Index.Table as TDzWideStrHashTable do
      begin
        if Find(KeyValue, @Item) then
        begin
          Delete(KeyValue);
          DeleteFromInserted(Item);
        end;
      end;
  end;
end;

procedure TDzPagedDataset.DeleteRecords(list: TDzPersistentList);
var
  i, j: Integer;
  Item: TDzPersistent;
  Index: PDzHashIndex;
begin
  if  IndexActive then
  begin
    for i := 0 to list.Count - 1 do
    begin
      Item := list[i];
      for j := 0 to fIndexes.Count - 1 do
      begin
        Index := PDzHashIndex(fIndexes[j]);
        case Index.DataType of
          dtInteger: (Index.Table as TDzIntegerRBTree).Delete(
            Item.Attributes[Index.Name]);
          dtAnsiStr: (Index.Table as TDzAnsiStrHashTable).Delete(
            Item.Attributes[Index.Name]);
          dtWideStr: (Index.Table as TDzWideStrHashTable).Delete(
            Item.Attributes[Index.Name]);
        end;
      end;
    end;
    Self.RecordDeleted(list);
  end;
end;

destructor TDzPagedDataset.Destroy;
begin
  Clear;
  FreeAndNil(fPages);
  FreeIndexes;
  FreeAndNil(fInserted);
  FreeAndNil(CriticalSection);
  inherited;
end;

procedure TDzPagedDataset.EnableIndex;
begin

end;

function TDzPagedDataset.FindIndex(IndexList: TList;
  const Name: TNameString): PDzHashIndex;
var
  i: Integer;
begin
  if Assigned(IndexList)  then
  for i := 0 to IndexList.Count - 1 do
  begin
    Result := PDzHashIndex(IndexList[i]);
    if Result.Name = Name then Exit;
  end;
  Result := nil; 
end;

function TDzPagedDataset.FindInsertedItem(const KeyField: TNameString;
  const KeyValue: Variant): TDzPersistent;
begin
  Result := Self.HashFindItem(fInsertedIndexes, KeyField, KeyValue);  
end;

function TDzPagedDataset.FindItem(const KeyField: TNameString;
  const KeyValue: Variant): TDzPersistent;
begin
  Result := Self.HashFindItem(fIndexes, KeyField, KeyValue);
end;

procedure TDzPagedDataset.ForcePages(StartIndex, Count: Integer;
  AbortOtherPages: Boolean);
var
  i: Integer;
begin
  Lock;
  try
    if AbortOtherPages then IntegerLinkList_Clear(fNeededPages);
    for i := StartIndex to StartIndex + Count - 1 do
    begin
      if not Assigned(Pages[i]) and not IntegerLinkList_Find(fLoadingPages, i) then
        IntegerLinkList_Insert(fNeededPages, i);
    end;
    LoadNeeded;
  finally
    Release;
  end;
end;

function TDzPagedDataset.GetItem(Index: Integer): TDzPersistent;
var
  list: TDzPersistentList;
begin
  list := Pages[Index div fPageSize];
  if list = nil then Result := nil
  else Result := list[Index mod fPageSize];
end;

function TDzPagedDataset.GetItemPageIndex(ItemIndex: Integer): Integer;
begin
  Result := ItemIndex div fPageSize;
end;

function TDzPagedDataset.GetPage(Index: Integer): TDzPersistentList;
begin
  CriticalSection.Acquire;
  try
    Result := TDzPersistentList(fPages[Index]);
  finally
    CriticalSection.Release;
  end;
end;

function TDzPagedDataset.HashFindItem(IndexList: TList; const KeyField: TNameString;
  const KeyValue: Variant): TDzPersistent;
var
  Index: PDzHashIndex;
  found: Boolean;
  value: Pointer;
begin
  Index := FindIndex(IndexList, KeyField);
  if Index = nil then Result := nil
  else begin
    case Index.DataType of
      dtInteger:
        begin
          value := TDzPersistent((Index.Table as TDzIntegerRBTree)[KeyValue]);
          found := value <> nil;
        end;
      dtAnsiStr: found := (Index.Table as TDzAnsiStrHashTable).Find(
        KeyValue, @value);
      dtWideStr: found := (Index.Table as TDzWideStrHashTable).Find(
        KeyValue, @value);
      else found := False;
    end;
    if found then Result := TDzPersistent(value)
    else Result := nil;
  end;
end;

function TDzPagedDataset.IndexActive: Boolean;
begin
  Result := Assigned(fIndexes) and (fIndexes.Count > 0); 
end;

procedure TDzPagedDataset.ItemDeleted(const KeyField: TNameString;
  const KeyValue: Variant);
var
  Index: PDzHashIndex;
begin
  Index := Self.FindIndex(fInsertedIndexes, KeyField);
  if Assigned(Index) then Self.DeleteInsertedItem(Index, KeyField, KeyValue);
end;

procedure TDzPagedDataset.LoadNeeded;
var
  pNode: PIntegerLink;
begin
  Lock;
  try
    if not Assigned(fLoadingPages) and Assigned(fNeededPages) then
    begin
      pNode := fNeededPages;  
      fNeededPages := fNeededPages.FLink;
      pNode.FLink := fLoadingPages;
      fLoadingPages := pNode;
      FetchPage(pNode.Value);
    end;
  finally
    Release;
  end;
end;

procedure TDzPagedDataset.Lock;
begin
  CriticalSection.Acquire;
end;

procedure TDzPagedDataset.ItemDeleted(Deled: TDzPersistent);
var
  Index: PDzHashIndex;
begin
  if Assigned(fInsertedIndexes) and (fInsertedIndexes.Count > 0) then
  begin
    Index := fInsertedIndexes[0];
    Self.DeleteInsertedItem(Index, Index.Name, Deled.Attributes[Index.Name]);
  end;
end;

procedure TDzPagedDataset.PageLoaded(PageIndex: Integer);
begin
  Lock;
  try
    IntegerLinkList_Delete(fLoadingPages, PageIndex);
  finally
    Release;
  end;
end;

function TDzPagedDataset.PageLoading(PageIndex: Integer): Boolean;
begin
  CriticalSection.Acquire;
  try
    Result := IntegerLinkList_Find(fLoadingPages, PageIndex);
  finally
    CriticalSection.Release;
  end;
end;

procedure TDzPagedDataset.RecordAdded(data: TDzPersistentList);
begin

end;

procedure TDzPagedDataset.RecordDeleted(data: TDzPersistentList);
begin

end;

procedure TDzPagedDataset.RefreshPage(PageIndex: Integer);
begin
  Pages[PageIndex] := nil;
  ForcePages(PageIndex, 1, False);
end;

procedure TDzPagedDataset.Release;
begin
  CriticalSection.Release;
end;

procedure TDzPagedDataset.SetItem(Index: Integer; const Value: TDzPersistent);
begin
end;

procedure TDzPagedDataset.SetPage(Index: Integer; const Value: TDzPersistentList);
var
  Page: TObject;
begin
  CriticalSection.Acquire;
  try
    Page := fPages[Index];
    if Page = Value then Exit;
    if Page = nil then
    begin
      fPages.Add(Index, Value);
      AddRecords(Value);
    end
    else begin
      DeleteRecords(TDzPersistentList(Page));
      Page.Free;
      if Value = nil then fPages.Delete(Index)
      else begin
        fPages[Index] := Value;
        AddRecords(Value);
      end;
    end;
  finally
    CriticalSection.Release;
  end;
end;

end.
